# -*- coding: utf-8 -*-
'''
@Software: PyCharm
@File    : search_weibo.py
@Author  : Bryan SHEN
@E-mail  : m18801919240_3@163.com
@Site    : Shanghai, China
@Time    : 2021-08-05
@Description:
'''

import pandas as pd
import time
from DataSearchRematchByRe.utils.elastic_search import ESPostWb, ESUserWb
import json
import re


class SeachDataWeibo(object):

    def __init__(self):
        pass

    def exclude_word_match(self, nwords):
        """ 输出排除词，返回列表not_clause """

        # print("排除词:" + nwords)
        not_clause = []
        if nwords != "":
            if "/" in nwords:
                word = nwords.split("/")
                not_clause = [{"match_phrase": {"content": i}} for i in word]
            elif "|" in nwords:
                word = nwords.split("|")
                not_clause = [{"match_phrase": {"content": i}} for i in word]
            else:
                word = nwords.split("￥")
                not_clause = [{"match_phrase": {"content": i}} for i in word]
        else:
            not_clause = not_clause

        return not_clause

    def must_word_match(self, mwords):
        """ 输出关键词，返回列表should_clause """

        if "/" in mwords:
            words = mwords.split("/")
            should_clause = [{"match_phrase": {"content": w}} for w in words]
        elif "|" in mwords:
            words = mwords.split("|")
            should_clause = [{"match_phrase": {"content": w}} for w in words]
        else:
            words = mwords.split("￥")
            should_clause = [{"match_phrase": {"content": w}} for w in words]

        return should_clause, words

    def query_data(self, k, mwords, words, should_clause, time, not_clause):
        """ 搜索数据，返回数据列表count """

        items = []
        if '/' in mwords:
            query = {
                # 筛选字段：若要在导出的结果中添加或删除字段，在这里操作
                "_source": ["user.user_id", "post_id", "post_create_time", "attitudes_count", "comments_count",
                            "reposts_count", "micro_task", "content", "source", "ad_state", "pic_num"],
                "query": {
                    "bool": {
                        "should": [
                            {
                                "bool": {"must": should_clause}
                            }
                        ],
                        "minimum_should_match": 1,
                        "must": [
                            {
                                "range": {
                                    "post_create_time": {
                                        "gte": time[0],
                                        "lte": time[1]
                                    }
                                }
                            }
                        ],
                        "must_not": not_clause
                    }
                }
            }

        elif '|' in mwords:
            query = {
                "_source": ["user.user_id", "post_id", "post_create_time", "attitudes_count", "comments_count",
                            "reposts_count", "micro_task", "content", "source", "ad_state", "pic_num"],
                "query": {
                    "bool": {
                        "should": should_clause,
                        "minimum_should_match": 1,
                        "must": [
                            {
                                "range": {
                                    "post_create_time": {
                                        "gte": time[0],
                                        "lte": time[1]
                                    }
                                }
                            }
                        ],
                        "must_not": not_clause
                    }
                }
            }

        else:
            query = {
                "_source": ["user.user_id", "post_id", "post_create_time", "attitudes_count", "comments_count",
                            "reposts_count", "micro_task", "content", "source", "ad_state", "pic_num" ],
                "query": {
                    "bool": {
                        "should": should_clause,
                        "minimum_should_match": 1,
                        "must": [
                            {
                                "range": {
                                    "post_create_time": {
                                        "gte": time[0],
                                        "lte": time[1]
                                    }
                                }
                            }
                        ],
                        "must_not": not_clause
                    }
                }
            }
        ep = ESPostWb.scan(query)
        for i2 in ep:
            i2['_source']['sum_match_keyword'] = k
            i2['_source']['match_keyword'] = mwords
            if 'user' in i2['_source']:
                i2['_source']['user_id'] = str(i2['_source']['user']['user_id'])
            if 'post_id' in i2['_source']:
                i2['_source']['post_id'] = str(i2['_source']['post_id'])
                i2['_source']['视频url'] = 'https://m.weibo.cn/status/' + i2['_source']['post_id']
            items.append(i2['_source'])
        # print(len(count1))

        return items

    # 处理数据
    def get_data(self, count):

        user_ids = []
        cou = []
        for i1 in count:
            if 'user_id' in i1:
                user_ids.append(i1['user_id'])
        user_ids = list(set(user_ids))

        count1 = []
        size = 2000
        for i9 in range(len(user_ids) // size + 1):
            body = {
                "size": size,
                "_source": [
                    "user_id",
                    "nickname",
                    "follower_count",
                    "region",
                    "city",
                    "gender",
                    "description",
                    "province",
                    "profile_url",
                    "post_price",
                    "forward_price",
                    "tags",
                    "cpm",
                    "verified_reason",
                    "account_type"
                ],
                "query": {
                    "terms": {
                        "user_id": user_ids[i9 * size: (i9 + 1) * size]
                    }
                }
            }
            eu = ESUserWb.search(body)
            for i3 in eu['hits']['hits']:
                if i3['_source'].get("user_id", ''):
                    i3['_source']['user_id'] = str(i3['_source']['user_id'])
                c = []
                if i3['_source'].get("tags"):
                    for z in i3['_source']['tags']:
                        if z.get('name'):
                            name = z['name'].split("$")
                            c.extend(name)
                i3['_source']['self_user_tags'] = c

                count1.append(i3['_source'])

        p = {i4['user_id']: i4 for i4 in count1}
        for i5 in count:
            o = {}
            o['target'] = ''
            if 'sum_match_keyword' in i5:
                o['sum_match_keyword'] = i5['sum_match_keyword']
            else:
                o['sum_match_keyword'] = ''
            if 'match_keyword' in i5:
                o['match_keyword'] = i5['match_keyword']
            else:
                o['match_keyword'] = ''
            if i5.get('user_id', ''):
                o['user_id'] = i5['user_id']
            else:
                o['user_id'] = ''
            if i5.get('post_id', ''):
                o['post_id'] = i5['post_id']
            else:
                o['post_id'] = ''
            if i5.get('content', ''):
                o['content'] = i5['content']
            else:
                o['content'] = ''
            if i5.get('post_create_time', ''):
                o['post_create_time'] = i5['post_create_time']
            else:
                o['post_create_time'] = ''
            if i5.get('视频url', ''):
                o['视频url'] = i5['视频url']
            else:
                o['视频url'] = ''
            if i5.get('attitudes_count', ''):
                o['点赞数'] = i5['attitudes_count']
            else:
                o['点赞数'] = ''
            if i5.get('comments_count', ''):
                o['评论数'] = i5['comments_count']
            else:
                o['评论数'] = ''
            if i5.get('reposts_count', ''):
                o['转发数'] = i5['reposts_count']
            else:
                o['转发数'] = ''
            if i5.get('micro_task', ''):
                o['micro_task'] = i5['micro_task']
            else:
                o['micro_task'] = ''
            if i5.get('ad_state', ''):
                o['ad_state'] = i5['ad_state']
            else:
                o['ad_state'] = ''
            if i5.get('source', ''):
                o['source'] = i5['source']
            else:
                o['source'] = ''
            if i5.get('pic_num', ''):
                o['pic_num'] = i5['pic_num']
            else:
                o['pic_num'] = ''
            if 'user_id' in i5:
                user = p.get(i5['user_id'], '')
                # print(user)
                if user != '':
                    if user.get('follower_count', ''):
                        o['follower_count'] = int(user['follower_count'])
                    else:
                        o['follower_count'] = ''
                if 'nickname' in user:
                    o['nickname'] = user['nickname']
                else:
                    o['nickname'] = ''
                if 'description' in user:
                    o['description'] = user['description']
                else:
                    o['description'] = ''
                if 'self_user_tags' in user:
                    o['self_user_tags'] = user['self_user_tags']
                else:
                    o['self_user_tags'] = ''
                if 'region' in user:
                    o['地域'] = user['region']
                else:
                    o['地域'] = ''
                if 'city' in user:
                    o['city'] = user['city']
                else:
                    o['city'] = ''
                if 'province' in user:
                    o['省份'] = user['province']
                else:
                    o['省份'] = ''
                if 'gender' in user:
                    o['性别'] = "男" if user['gender'] == "m" else "女"
                else:
                    o['性别']=''
                if 'profile_url' in user:
                    o['主页链接'] = 'https://weibo.com/' + user['profile_url']
                else:
                    o['主页链接'] = ''
                if 'post_price' in user:
                    o['原创价格'] = user['post_price']
                else:
                    o['原创价格'] = ''
                if 'forward_price' in user:
                    o['转发价格'] = user['forward_price']
                else:
                    o['转发价格'] = ''
                if 'account_type' in user:
                    o['account_type'] = user['account_type']
                else:
                    o['account_type'] = ''
                if 'cpm' in user:
                    o['cpm'] = user['cpm']
                else:
                    o['cpm'] = ''
                if 'verified_reason' in user:
                    o['verified_reason'] = user['verified_reason']
                else:
                    o['verified_reason'] = ''
                cou.append(o)

        cou = pd.DataFrame(cou)
        cou = cou.drop_duplicates(subset=['sum_match_keyword', 'post_id'], keep='first')  # 删除重复项的函数, subset表示要删除的重复项指定列
        cou = cou.rename(columns={"self_user_tags": "tag"})  # 修改列名

        # cou = cou[~cou['follower_count'].isnull()]  # 删除粉丝数为空的
        # cou = cou[cou['follower_count'] >= 10000]
        cou['follower_count'].replace('', 0, inplace=True)
        items = cou.to_dict(orient='records')

        print("items: ", len(items))

        return items

    def re_match(self, items, kw1_kw2_dic):
        """ kw1前后5个字以内出现kw2, 算匹配成功 """

        new_items = []

        for item in items:
            matched_items = []
            text = item["content"]
            for k1, k2 in kw1_kw2_dic.items():
                for kw in k2:
                    pattern1 = re.compile(k1 + r'[^，。,.】·：；;▪+、➕\n\t\r]{0,5}?' + kw)
                    matched_items1 = re.findall(pattern1, str(text).lower())
                    pattern2 = re.compile(kw + r'[^，。,.】·：；;▪+、➕\n\t\r]{0,5}?' + k1)
                    matched_items2 = re.findall(pattern2, str(text).lower())
                    if matched_items1:
                        matched_items.extend(matched_items1)
                    if matched_items2:
                        matched_items.extend(matched_items2)
            if len(matched_items) > 0:
                item["re_matched_item"] = matched_items[0]
                new_items.append(item)

        print("new_items: ", len(new_items))

        items = pd.DataFrame(items)
        new_items = pd.DataFrame(new_items)

        columns1 = ["match_keyword", "user_id", "red_id", "follower_num", "nickname", "总获赞", "总收藏", "笔记数",
            "地域", "sex", "up_description", "主页链接", "post_id", "title", "content", "keywords", "categories", "ocr",
            "点赞数", "收藏数", "分享数", "评论数", "视频url", "post_create_time", "name", "post_type", "tags", "belongMcn",
            "picturePrice", "videoPrice", "target", "official_verified", "topics_name", "red_official_verify_content",
            "data_type", "sum_match_keyword"]
        columns2 = ["match_keyword", "user_id", "red_id", "follower_num", "nickname", "总获赞", "总收藏", "笔记数",
            "地域", "sex", "up_description", "主页链接", "post_id", "title", "content", "keywords", "categories", "ocr",
            "点赞数", "收藏数", "分享数", "评论数", "视频url", "post_create_time", "name", "post_type", "tags", "belongMcn",
            "picturePrice", "videoPrice", "target", "official_verified", "topics_name", "red_official_verify_content",
            "data_type", "sum_match_keyword", "re_matched_item"]

        with pd.ExcelWriter("weibo_match_test.xlsx", engine='xlsxwriter', options={'strings_to_urls': False}) as writer:
            items.to_excel(writer, sheet_name='sheet1', index=False, columns=columns1)
            new_items.to_excel(writer, sheet_name="sheet2", index=False, columns=columns2)

    def run(self, excluded_words_string, must_words_string, start_date, end_date, kw1_kw2_dic):

        # 获取 "排除条件"
        not_clause = self.exclude_word_match(excluded_words_string)
        # 获取 "匹配条件"
        should_clause, words = self.must_word_match(must_words_string)
        # 根据 排除条件 和 匹配条件 筛选数据
        count = self.query_data("xxxxx", must_words_string, words, should_clause, [start_date, end_date], not_clause)
        items = self.get_data(count)

        self.re_match(items, kw1_kw2_dic)


if __name__ == "__main__":

    excluded_words_string = ""

    must_words_string = "lancome|兰蔻"

    start_date = "2021-08-25"
    end_date = "2021-08-26"

    kw1_kw2_dic = {
        "兰蔻": "精华",
        "粉底液": "推荐",
        "面霜": "美白"
    }

    s = SeachDataWeibo()

    s.run(excluded_words_string, must_words_string, start_date, end_date, kw1_kw2_dic)


